'''
label_reduce
remove label_freq lower than 500
'''
fin = open("../dataProcess/data/user_info.txt")
fin_train = open("./ua.base.example.purified.multi")
fin_test = open("./ua.test.example.multi")
fout_train = open("./base.reduce.txt", "w")
fou_test = open("./test.reduce.txt", "w")

label_dic = {}
label_remain = set()
for line in fin:
	line = line.strip()
	line_list = line.split("\t" or " ")
	labels = line_list[1].split("/")
	for l in labels:
		if int(l) not in label_dic:
			label_dic[int(l)] = 1
		else:
			label_dic[int(l)] = label_dic[int(l)] + 1
label_list = label_dic.items()
print label_list
label_list = sorted(label_list, key = lambda pair : pair[1], reverse = True)
print label_list

for pair in label_list:
	if pair[1] > 500:
		label_remain.add(pair[0])

for line in fin_train:
	line = line.strip()
	line_list = line.split(" ")
	fnum = int(line_list[2])
	flist = []
	for i in range(5, 5+fnum):
		f = int(line_list[i].split(":")[0])
		if f in label_remain:
			flist.append(f)
	line_out = line_list[0] + " " + line_list[1] + " " + str(len(flist) + 1) + " " + line_list[3] + " " + line_list[4]
	for f in flist:
		line_out = line_out + " " +  str(f) + ":1"
	line_out = line_out + " " + line_list[-2] + " " + line_list[-1]
	fout_train.write(line_out + "\n")

for line in fin_test:
	line = line.strip()
	line_list = line.split(" ")
	fnum = int(line_list[2])
	flist = []
	for i in range(5, 5+fnum):
		f = int(line_list[i].split(":")[0])
		if f in label_remain:
			flist.append(f)
	line_out = line_list[0] + " " + line_list[1] + " " + str(len(flist) + 1) + " " + line_list[3] + " " + line_list[4]
	for f in flist:
		line_out = line_out + " " +  str(f) + ":1"
	line_out = line_out + " " + line_list[-2] + " " + line_list[-1]
	fou_test.write(line_out + "\n")